{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "flower_type_pred_v7.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "abbfde55ec614d0296644f4da4726fb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7445d3ba971f411999dcb8d5152804be",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f44503b8f53f41099b547f3efe4e3fe7",
              "IPY_MODEL_dc8f3db3db094ab18754d872ed67c23d"
            ]
          }
        },
        "7445d3ba971f411999dcb8d5152804be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f44503b8f53f41099b547f3efe4e3fe7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_64fb11765a964626b7a5250d946aa835",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_50e0a23481bb499fa05ad75de3ae5a94"
          }
        },
        "dc8f3db3db094ab18754d872ed67c23d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0248f736e97447eea744ccd6c4fef90f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [01:42&lt;00:00, 20.51s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_762d7252111c467088405b8d4d4fef8b"
          }
        },
        "64fb11765a964626b7a5250d946aa835": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "50e0a23481bb499fa05ad75de3ae5a94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0248f736e97447eea744ccd6c4fef90f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "762d7252111c467088405b8d4d4fef8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0d492bbfb394e32917696fd734418c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2b48fc6a4b504cfd9d625d045c09ed45",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5e983cb79ae54cb7a062fa88b69f15fc",
              "IPY_MODEL_f04ec04c934143aa8e6e7fa936a3b39b"
            ]
          }
        },
        "2b48fc6a4b504cfd9d625d045c09ed45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e983cb79ae54cb7a062fa88b69f15fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_03a658063e7d40519fd8edc5393e32bf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 7,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 7,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e91ca3007014570adc27b5177ddb903"
          }
        },
        "f04ec04c934143aa8e6e7fa936a3b39b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_08853417d55940ff9050078317710b0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 7/7 [00:00&lt;00:00,  7.76it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03b0f630637d43f19e85848a9e7d9520"
          }
        },
        "03a658063e7d40519fd8edc5393e32bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e91ca3007014570adc27b5177ddb903": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08853417d55940ff9050078317710b0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03b0f630637d43f19e85848a9e7d9520": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "invb01u8bdOv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ver = 'v7'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VfcmbjtpVLW2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import *\n",
        "from sklearn.preprocessing import *\n",
        "from sklearn.metrics import *\n",
        "from lightgbm import LGBMClassifier\n",
        "from xgboost import XGBClassifier\n",
        "\n",
        "import gc\n",
        "from tqdm import *"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zln8E9QHVcyl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = pd.read_csv('Train.csv')\n",
        "test = pd.read_csv('Test.csv')\n",
        "ss = pd.read_csv('sample_submission.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0qBLg1oV4ML",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.concat([train, test]).reset_index(drop = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9KtHa-vCsvoV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 160
        },
        "outputId": "6c1d5f9f-388d-4d84-dfac-845772652aef"
      },
      "source": [
        "df.nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Area_Code        30900\n",
              "Locality_Code       22\n",
              "Region_Code       2185\n",
              "Height             110\n",
              "Diameter           151\n",
              "Class                8\n",
              "Species            263\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZxbJMIWs2U_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d8f3715b-7bcf-4b79-d38d-e24cf01f6c01"
      },
      "source": [
        "df['Region_Species'] = pd.factorize(df['Region_Code'].astype('str') + df['Species'].astype('str'))[0]\n",
        "df['Region_Species'].nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17667"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mtQT91gocqaE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "cee318e3-e8cf-4885-f7bc-22885a39667b"
      },
      "source": [
        "df['height*diameter'] = df['Height'] * df['Diameter']\n",
        "df['height/diameter'] = df['Height'] / df['Diameter']\n",
        "df['log_diameter'] = np.log(df['Diameter'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pandas/core/series.py:679: RuntimeWarning: divide by zero encountered in log\n",
            "  result = getattr(ufunc, method)(*inputs, **kwargs)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_HkJYbic6-Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120,
          "referenced_widgets": [
            "abbfde55ec614d0296644f4da4726fb1",
            "7445d3ba971f411999dcb8d5152804be",
            "f44503b8f53f41099b547f3efe4e3fe7",
            "dc8f3db3db094ab18754d872ed67c23d",
            "64fb11765a964626b7a5250d946aa835",
            "50e0a23481bb499fa05ad75de3ae5a94",
            "0248f736e97447eea744ccd6c4fef90f",
            "762d7252111c467088405b8d4d4fef8b"
          ]
        },
        "outputId": "eb56e1d0-78cd-47a7-db3a-cd45eb16fba7"
      },
      "source": [
        "cat_fts = ['Area_Code', 'Locality_Code', 'Region_Code', 'Species', 'Region_Species']\n",
        "num_fts = ['Height', 'Diameter','height*diameter', 'height/diameter', 'log_diameter']\n",
        "\n",
        "for g in tqdm_notebook(cat_fts):\n",
        "\n",
        "  grp = df.groupby(g)[num_fts].mean()\n",
        "  grp.columns = [c + f'_grpd_by_{g}_mean' for c in grp.columns]\n",
        "  df = pd.merge(df, grp, on = g, how = 'left')\n",
        "\n",
        "  grp = df.groupby(g)[num_fts].std()\n",
        "  grp.columns = [c + f'_grpd_by_{g}_std' for c in grp.columns]\n",
        "  df = pd.merge(df, grp, on = g, how = 'left')\n",
        "\n",
        "  grp = df.groupby(g)[num_fts].min()\n",
        "  grp.columns = [c + f'_grpd_by_{g}_min' for c in grp.columns]\n",
        "  df = pd.merge(df, grp, on = g, how = 'left')\n",
        "\n",
        "  grp = df.groupby(g)[num_fts].max()\n",
        "  grp.columns = [c + f'_grpd_by_{g}_max' for c in grp.columns]\n",
        "  df = pd.merge(df, grp, on = g, how = 'left')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "abbfde55ec614d0296644f4da4726fb1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QfJ8wl58W2Hz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120,
          "referenced_widgets": [
            "a0d492bbfb394e32917696fd734418c5",
            "2b48fc6a4b504cfd9d625d045c09ed45",
            "5e983cb79ae54cb7a062fa88b69f15fc",
            "f04ec04c934143aa8e6e7fa936a3b39b",
            "03a658063e7d40519fd8edc5393e32bf",
            "9e91ca3007014570adc27b5177ddb903",
            "08853417d55940ff9050078317710b0f",
            "03b0f630637d43f19e85848a9e7d9520"
          ]
        },
        "outputId": "b08cea0c-688c-4736-e3d3-c5368670b15b"
      },
      "source": [
        "fts = ['Area_Code','Locality_Code','Region_Code','Height','Diameter', 'Species', 'Region_Species']\n",
        "for g in tqdm_notebook(fts):\n",
        "  rf = [c for c in fts if c != g]\n",
        "  grp = df.groupby(g)[rf].nunique()\n",
        "  grp.columns = [c + f'_grpd_by_{g}_nunique' for c in grp.columns]\n",
        "  df = pd.merge(df, grp, on = g, how = 'left')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a0d492bbfb394e32917696fd734418c5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=7.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLmGNPFObn2N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for f in fts:\n",
        "  df[f + '_freq'] = df[f].map(df[f].value_counts(normalize = True))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jSdWOa2hX7Uc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train, test = df[:train.shape[0]].reset_index(drop = True), df[train.shape[0]:].reset_index(drop = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ag23_zOcXvxP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "features = [c for c in train.columns if c not in ['Class', \"Area_Code\", \"Region_Code\", 'Region_Species']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IqV_PDBWpvBb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0df11d62-5480-4592-fa0c-29ea51be6ba3"
      },
      "source": [
        "len(features)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "156"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BchuAjlmWnFd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "b5c64a71-b397-4ffe-ce86-1f70d58e4a6b"
      },
      "source": [
        "oofs = np.zeros((len(train), 8))\n",
        "preds = np.zeros((len(test), 8))\n",
        "N_SPLITS = 15\n",
        "folds = StratifiedKFold(N_SPLITS, shuffle = True, random_state = 77)\n",
        "\n",
        "for fold_, (trn_idx, val_idx) in enumerate(folds.split(train, train['Class'])):\n",
        "\n",
        "  print(f'\\n\\n Fold {fold_} \\\\n\\n')\n",
        "  X_trn, y_trn = train[features].iloc[trn_idx], train[\"Class\"][trn_idx]\n",
        "  X_val, y_val = train[features].iloc[val_idx], train[\"Class\"][val_idx]\n",
        "\n",
        "  clf = XGBClassifier(n_estimators = 2000, learning_rate = 0.03, colsample_bytree = 0.6, colsample_bynode=0.6, reg_alpha = 0.5, reg_lambda = 4, random_state = 2,\n",
        "                      max_depth = 16, tree_method = 'gpu_hist')\n",
        "\n",
        "  clf.fit(X_trn, y_trn, eval_set = [(X_val, y_val)], early_stopping_rounds = 200, verbose = 50, eval_metric = 'mlogloss')\n",
        "\n",
        "  oofs[val_idx] = clf.predict_proba(X_val)\n",
        "  preds += clf.predict_proba(test[features]) / N_SPLITS\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Fold 0 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.0218\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.0492\n",
            "[100]\tvalidation_0-mlogloss:0.846962\n",
            "[150]\tvalidation_0-mlogloss:0.784406\n",
            "[200]\tvalidation_0-mlogloss:0.76648\n",
            "[250]\tvalidation_0-mlogloss:0.762545\n",
            "[300]\tvalidation_0-mlogloss:0.766154\n",
            "[350]\tvalidation_0-mlogloss:0.77301\n",
            "[400]\tvalidation_0-mlogloss:0.780924\n",
            "[450]\tvalidation_0-mlogloss:0.787922\n",
            "Stopping. Best iteration:\n",
            "[251]\tvalidation_0-mlogloss:0.762499\n",
            "\n",
            "\n",
            "\n",
            " Fold 1 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02554\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.03854\n",
            "[100]\tvalidation_0-mlogloss:0.828675\n",
            "[150]\tvalidation_0-mlogloss:0.765145\n",
            "[200]\tvalidation_0-mlogloss:0.748275\n",
            "[250]\tvalidation_0-mlogloss:0.745883\n",
            "[300]\tvalidation_0-mlogloss:0.750678\n",
            "[350]\tvalidation_0-mlogloss:0.757734\n",
            "[400]\tvalidation_0-mlogloss:0.766202\n",
            "[450]\tvalidation_0-mlogloss:0.775004\n",
            "Stopping. Best iteration:\n",
            "[250]\tvalidation_0-mlogloss:0.745883\n",
            "\n",
            "\n",
            "\n",
            " Fold 2 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.0224\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.00356\n",
            "[100]\tvalidation_0-mlogloss:0.784306\n",
            "[150]\tvalidation_0-mlogloss:0.713704\n",
            "[200]\tvalidation_0-mlogloss:0.689511\n",
            "[250]\tvalidation_0-mlogloss:0.683474\n",
            "[300]\tvalidation_0-mlogloss:0.684904\n",
            "[350]\tvalidation_0-mlogloss:0.68977\n",
            "[400]\tvalidation_0-mlogloss:0.696702\n",
            "[450]\tvalidation_0-mlogloss:0.703563\n",
            "Stopping. Best iteration:\n",
            "[254]\tvalidation_0-mlogloss:0.683215\n",
            "\n",
            "\n",
            "\n",
            " Fold 3 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02422\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.03726\n",
            "[100]\tvalidation_0-mlogloss:0.824035\n",
            "[150]\tvalidation_0-mlogloss:0.754023\n",
            "[200]\tvalidation_0-mlogloss:0.731793\n",
            "[250]\tvalidation_0-mlogloss:0.725902\n",
            "[300]\tvalidation_0-mlogloss:0.727945\n",
            "[350]\tvalidation_0-mlogloss:0.734282\n",
            "[400]\tvalidation_0-mlogloss:0.739859\n",
            "[450]\tvalidation_0-mlogloss:0.745882\n",
            "Stopping. Best iteration:\n",
            "[259]\tvalidation_0-mlogloss:0.725568\n",
            "\n",
            "\n",
            "\n",
            " Fold 4 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02264\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.00811\n",
            "[100]\tvalidation_0-mlogloss:0.791623\n",
            "[150]\tvalidation_0-mlogloss:0.725402\n",
            "[200]\tvalidation_0-mlogloss:0.702405\n",
            "[250]\tvalidation_0-mlogloss:0.697487\n",
            "[300]\tvalidation_0-mlogloss:0.699617\n",
            "[350]\tvalidation_0-mlogloss:0.70477\n",
            "[400]\tvalidation_0-mlogloss:0.71087\n",
            "[450]\tvalidation_0-mlogloss:0.718276\n",
            "Stopping. Best iteration:\n",
            "[258]\tvalidation_0-mlogloss:0.697118\n",
            "\n",
            "\n",
            "\n",
            " Fold 5 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02125\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.02999\n",
            "[100]\tvalidation_0-mlogloss:0.821718\n",
            "[150]\tvalidation_0-mlogloss:0.75975\n",
            "[200]\tvalidation_0-mlogloss:0.742641\n",
            "[250]\tvalidation_0-mlogloss:0.740722\n",
            "[300]\tvalidation_0-mlogloss:0.7456\n",
            "[350]\tvalidation_0-mlogloss:0.752851\n",
            "[400]\tvalidation_0-mlogloss:0.759514\n",
            "Stopping. Best iteration:\n",
            "[235]\tvalidation_0-mlogloss:0.740018\n",
            "\n",
            "\n",
            "\n",
            " Fold 6 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02296\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.02193\n",
            "[100]\tvalidation_0-mlogloss:0.805042\n",
            "[150]\tvalidation_0-mlogloss:0.736835\n",
            "[200]\tvalidation_0-mlogloss:0.714024\n",
            "[250]\tvalidation_0-mlogloss:0.707432\n",
            "[300]\tvalidation_0-mlogloss:0.709067\n",
            "[350]\tvalidation_0-mlogloss:0.713469\n",
            "[400]\tvalidation_0-mlogloss:0.719444\n",
            "Stopping. Best iteration:\n",
            "[249]\tvalidation_0-mlogloss:0.70737\n",
            "\n",
            "\n",
            "\n",
            " Fold 7 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02639\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.0703\n",
            "[100]\tvalidation_0-mlogloss:0.879731\n",
            "[150]\tvalidation_0-mlogloss:0.830823\n",
            "[200]\tvalidation_0-mlogloss:0.820015\n",
            "[250]\tvalidation_0-mlogloss:0.822817\n",
            "[300]\tvalidation_0-mlogloss:0.831741\n",
            "[350]\tvalidation_0-mlogloss:0.841798\n",
            "[400]\tvalidation_0-mlogloss:0.85223\n",
            "Stopping. Best iteration:\n",
            "[212]\tvalidation_0-mlogloss:0.819419\n",
            "\n",
            "\n",
            "\n",
            " Fold 8 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02539\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.01945\n",
            "[100]\tvalidation_0-mlogloss:0.806278\n",
            "[150]\tvalidation_0-mlogloss:0.743097\n",
            "[200]\tvalidation_0-mlogloss:0.724118\n",
            "[250]\tvalidation_0-mlogloss:0.722871\n",
            "[300]\tvalidation_0-mlogloss:0.727947\n",
            "[350]\tvalidation_0-mlogloss:0.734806\n",
            "[400]\tvalidation_0-mlogloss:0.742887\n",
            "Stopping. Best iteration:\n",
            "[231]\tvalidation_0-mlogloss:0.721991\n",
            "\n",
            "\n",
            "\n",
            " Fold 9 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02499\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.05575\n",
            "[100]\tvalidation_0-mlogloss:0.856012\n",
            "[150]\tvalidation_0-mlogloss:0.796634\n",
            "[200]\tvalidation_0-mlogloss:0.781088\n",
            "[250]\tvalidation_0-mlogloss:0.779353\n",
            "[300]\tvalidation_0-mlogloss:0.78606\n",
            "[350]\tvalidation_0-mlogloss:0.794053\n",
            "[400]\tvalidation_0-mlogloss:0.804401\n",
            "Stopping. Best iteration:\n",
            "[240]\tvalidation_0-mlogloss:0.778303\n",
            "\n",
            "\n",
            "\n",
            " Fold 10 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02336\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.01773\n",
            "[100]\tvalidation_0-mlogloss:0.809875\n",
            "[150]\tvalidation_0-mlogloss:0.749737\n",
            "[200]\tvalidation_0-mlogloss:0.73407\n",
            "[250]\tvalidation_0-mlogloss:0.733435\n",
            "[300]\tvalidation_0-mlogloss:0.73907\n",
            "[350]\tvalidation_0-mlogloss:0.747264\n",
            "[400]\tvalidation_0-mlogloss:0.75623\n",
            "Stopping. Best iteration:\n",
            "[221]\tvalidation_0-mlogloss:0.732195\n",
            "\n",
            "\n",
            "\n",
            " Fold 11 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02542\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.04426\n",
            "[100]\tvalidation_0-mlogloss:0.835267\n",
            "[150]\tvalidation_0-mlogloss:0.773005\n",
            "[200]\tvalidation_0-mlogloss:0.754469\n",
            "[250]\tvalidation_0-mlogloss:0.753144\n",
            "[300]\tvalidation_0-mlogloss:0.759853\n",
            "[350]\tvalidation_0-mlogloss:0.767941\n",
            "[400]\tvalidation_0-mlogloss:0.776358\n",
            "Stopping. Best iteration:\n",
            "[233]\tvalidation_0-mlogloss:0.75219\n",
            "\n",
            "\n",
            "\n",
            " Fold 12 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.0245\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.00988\n",
            "[100]\tvalidation_0-mlogloss:0.797492\n",
            "[150]\tvalidation_0-mlogloss:0.734568\n",
            "[200]\tvalidation_0-mlogloss:0.714782\n",
            "[250]\tvalidation_0-mlogloss:0.710353\n",
            "[300]\tvalidation_0-mlogloss:0.714056\n",
            "[350]\tvalidation_0-mlogloss:0.719939\n",
            "[400]\tvalidation_0-mlogloss:0.726147\n",
            "Stopping. Best iteration:\n",
            "[240]\tvalidation_0-mlogloss:0.710113\n",
            "\n",
            "\n",
            "\n",
            " Fold 13 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02417\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.04183\n",
            "[100]\tvalidation_0-mlogloss:0.830209\n",
            "[150]\tvalidation_0-mlogloss:0.766781\n",
            "[200]\tvalidation_0-mlogloss:0.749353\n",
            "[250]\tvalidation_0-mlogloss:0.74824\n",
            "[300]\tvalidation_0-mlogloss:0.754395\n",
            "[350]\tvalidation_0-mlogloss:0.76306\n",
            "[400]\tvalidation_0-mlogloss:0.772203\n",
            "Stopping. Best iteration:\n",
            "[228]\tvalidation_0-mlogloss:0.747058\n",
            "\n",
            "\n",
            "\n",
            " Fold 14 \\n\n",
            "\n",
            "[0]\tvalidation_0-mlogloss:2.02499\n",
            "Will train until validation_0-mlogloss hasn't improved in 200 rounds.\n",
            "[50]\tvalidation_0-mlogloss:1.04234\n",
            "[100]\tvalidation_0-mlogloss:0.835038\n",
            "[150]\tvalidation_0-mlogloss:0.772818\n",
            "[200]\tvalidation_0-mlogloss:0.752614\n",
            "[250]\tvalidation_0-mlogloss:0.748826\n",
            "[300]\tvalidation_0-mlogloss:0.752198\n",
            "[350]\tvalidation_0-mlogloss:0.758549\n",
            "[400]\tvalidation_0-mlogloss:0.765899\n",
            "[450]\tvalidation_0-mlogloss:0.773489\n",
            "Stopping. Best iteration:\n",
            "[252]\tvalidation_0-mlogloss:0.748499\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qT1rSXSha9BP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "144ff194-795c-4d0f-ae15-7755b95c9c36"
      },
      "source": [
        "oof_val_score = log_loss(train['Class'], oofs)\n",
        "print(f'Final Log loss: {oof_val_score}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Final Log loss: 0.7380900319264871\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mJN9Uq2CaDYa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ss[ss.columns.tolist()] = preds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eMKXozZFaK4s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "97b3c3d5-d0fa-4836-c8db-7842d743769d"
      },
      "source": [
        "SUB_FILE_NAME = f'mft_{ver}.csv'\n",
        "ss.to_csv(SUB_FILE_NAME, index=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_641a608b-000e-422b-99f3-659e7eec48e8\", \"mft_v7.csv\", 4902349)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}